package org.clockwise.driver;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;

import org.clockwise.feature.Account;
import org.clockwise.feature.AccountPool;
import org.clockwise.feature.Query;
import org.clockwise.feature.QueryContext;
import org.clockwise.multimethod.PageRank;
import org.clockwise.srw.AccountRelationPool;
import org.clockwise.util.DocReader;
import org.clockwise.util.MyFileReader;
import org.clockwise.util.MyFileWriter;
import org.clockwise.util.MyStrUtil;

public class CorpusExtractionDriver {

	public static String cleanDocName = "cleanDoc.txt";

	public static void generateQueryDataFile() {
		HashMap<Long, Account> map = AccountOperationDriver.getAccount();
		System.out.println("User dict ready");
		AccountPool ap = new AccountPool(map);
		System.out.println("AccountPool ready");
		List<QueryContext> contextList = CorpusExtractionDriver
				.getQueryContextFromText(ap);
//		List<QueryContext> contextList = CorpusExtractionDriver
//				.getQueryContext(ap);
		System.out.println("QueryContext list ready");
		// globalNormalize(contextList);
		// System.out.println("Global normalization done");
		MyFileWriter mfw = new MyFileWriter(LinkerDriver.DATA_DIR
				+ "pureData.txt", true);

		Iterator<QueryContext> iter = contextList.iterator();
		int count = 0;
		while (iter.hasNext()) {
			recordQueryContext(iter.next(), count, mfw);
			count++;
		}
		mfw.close();
		recordContext(contextList);
		
		System.out.println("Record pure data done");
		
//		List<List<List<double[]>>> data = new ArrayList<List<List<double[]>>>();
//		List<List<List<Long>>> dict = new ArrayList<List<List<Long>>>();
//		List<double[][]> relation = new ArrayList<double[][]>();
//		List<double[][]> feature = new ArrayList<double[][]>();
//		List<int[]> ruler = new ArrayList<int[]>();

//		int all = DataReader.readData(data, dict);
//		AccountRelationPool.getRelations(data, dict, feature, relation, ruler);
		
//		recordMatrix(relation);
		
//		System.out.println("Record matrix done");

	}

	public static List<List<List<String>>> getVerbose() {
		MyFileReader mfr = new MyFileReader(LinkerDriver.DATA_DIR
				+ "verbose.txt");
		List<List<List<String>>> ver = new ArrayList<List<List<String>>>();
		String temp = null;
		int contextId = -1, queryId = -1;
		String info = null, stat = null;
		List<String> sl = new ArrayList<String>();
		List<List<String>> sll = new ArrayList<List<String>>();
		String title = null;

		while ((temp = mfr.getNextLine()) != null) {
			if (temp.contains("Query")) {
				String[] seg = temp.split("\t");
				assert (seg.length >= 3);
				int ci = Integer.parseInt(seg[0].substring("Context".length()));
				int qi = Integer.parseInt(seg[1].substring("Query".length()));
				title = seg[2];
				
				if (ci != contextId) {
					contextId = ci;
					queryId = qi;
					if (sl.size() > 0) {
						sll.add(sl);
//						System.out.println("Context " + contextId + " Query "
//								+ queryId + " : " + sl.size());
					}
					sl = new ArrayList<String>();
//					System.out.println(contextId + " : " + queryId);
					if (sll.size() > 0) {
						ver.add(sll);
//						System.out.println("Context " + contextId + " : " + (sll.size()));
					}
					sll = new ArrayList<List<String>>();
				} else {
					if (qi != queryId) {
						queryId = qi;
						if (sl.size() > 0) {
							sll.add(sl);
//							System.out.println("Context " + contextId + " Query "
//									+ queryId + " : " + sl.size());
						}
						sl = new ArrayList<String>();
//						System.out.println(contextId + " : " + queryId);
					}
				}
			} else {
				sl.add(title + "\t" + temp + "\n" + mfr.getNextLine());
			}
		}
		return ver;
	}

	public static void recordContext(List<QueryContext> contextList) {
		MyFileWriter mfw = new MyFileWriter(LinkerDriver.DATA_DIR
				+ "verbose.txt", true);
		for (int i = 0; i < contextList.size(); i++) {
			QueryContext qc = contextList.get(i);
			for (int j = 0; j < qc.queryList.size(); j++) {
				Query q = qc.queryList.get(j);
				mfw.write("Context" + i + "\tQuery" + j + "\t<" + q.keyword.key
						+ ">\n");
				for (int k = 0; k < q.potential.size(); k++) {
					mfw.write("\t" + q.potential.get(k).toString() + "\n");
					double[] f = q.featureList.get(k);
					StringBuilder sb = new StringBuilder();
					for (int d = 0; d < f.length; d++) {
						sb.append("\t");
						sb.append(f[d]);
					}
					mfw.write("\t" + sb.toString() + "\n");
				}
			}
		}
		mfw.close();
	}
	
	public static void recordMatrix(List<double[][]> contextList) {
		MyFileWriter mfw = new MyFileWriter(LinkerDriver.DATA_DIR + "matrix.txt", true);
		int cid = 0;
		Iterator<double[][]> mIter = contextList.iterator();
		while (mIter.hasNext()) {
			mfw.write("Context" + cid + "\n");
			StringBuilder sb = new StringBuilder();
			double[][] matrix = mIter.next();
			for (int i = 0; i < matrix.length; i++) {
				for (int j = 0; j < matrix[i].length; j++) {
					sb.append(matrix[i][j] + "\t");
				}
				sb.append("\n");
			}
			mfw.write(sb.toString() + "\n");
			cid++;
		}
	}

	public static void globalNormalize(List<QueryContext> contextList) {
		double[] max = null;
		Iterator<QueryContext> iter = contextList.iterator();
		while (iter.hasNext()) {
			double[] submax = iter.next().getMaxFeature();
			if (max == null) {
				max = new double[submax.length];
			}
			for (int i = 0; i < max.length; i++) {
				if (max[i] < submax[i]) {
					max[i] = submax[i];
				}
			}
		}

		for (int i = 0; i < max.length; i++) {
			System.out.print(max[i] + "\t");
		}
		System.out.println("");

		for (int i = 0; i < max.length; i++) {
			if (max[i] == 0) {
				System.out.println("Fatal Error! Failed to normalize!");
				return;
			}
		}

		iter = contextList.iterator();
		while (iter.hasNext()) {
			iter.next().normalize(max);
		}
	}

	/*
	 * public static void globalNormalize(List<List<List<double[]>>> data) {
	 * double[] maxFeature = new double[PageRank.weight.length];
	 * Iterator<List<List<double[]>>> contextIter = data.iterator(); while
	 * (contextIter.hasNext()) { List<List<double[]>> context =
	 * contextIter.next(); Iterator<List<double[]>> queryIter =
	 * context.iterator(); while (queryIter.hasNext()) { List<double[]> query =
	 * queryIter.next(); Iterator<double[]> userIter = query.iterator(); while
	 * (userIter.hasNext()) { double[] feature = userIter.next(); for (int i =
	 * 0; i < maxFeature.length; i++) { if (maxFeature[i] < feature[i]) {
	 * maxFeature[i] = feature[i]; } } } } } for (int i = 0; i <
	 * maxFeature.length; i++) { if (maxFeature[i] == 0) {
	 * System.out.println("Fatal Error! Failed to execute global normalization!"
	 * ); return; } } contextIter = data.iterator(); while
	 * (contextIter.hasNext()) { List<List<double[]>> context =
	 * contextIter.next(); Iterator<List<double[]>> queryIter =
	 * context.iterator(); while (queryIter.hasNext()) { List<double[]> query =
	 * queryIter.next(); Iterator<double[]> userIter = query.iterator(); while
	 * (userIter.hasNext()) { double[] feature = userIter.next(); for (int i =
	 * 0; i < maxFeature.length; i++) { feature[i] /= maxFeature[i]; } } } }
	 * 
	 * }
	 */
	public static void recordQueryContext(QueryContext context, int contextId,
			MyFileWriter mfw) {
		List<Query> queryList = context.queryList;
		int count = 0;
		Iterator<Query> iter = queryList.iterator();
		while (iter.hasNext()) {
			recordQuery(iter.next(), contextId, count, mfw);
			count++;
		}
	}

	public static void recordQuery(Query query, int contextId, int queryId,
			MyFileWriter mfw) {
		String prefix = "" + contextId + "\t" + queryId + "\t";
		Iterator<Account> userIter = query.potential.iterator();
		Iterator<double[]> iter = query.featureList.iterator();
		while (iter.hasNext()) {
			StringBuilder sb = new StringBuilder();
			sb.append(prefix);
			if (userIter.hasNext()) {
				sb.append(userIter.next().id);
			} else {
				System.out.println("Fatal Error! Wrong potential list!");
			}
			double[] feature = iter.next();
			for (int i = 0; i < feature.length; i++) {
				sb.append("\t");
				sb.append(feature[i]);
			}
			sb.append("\n");
			mfw.write(sb.toString());
		}
		if (userIter.hasNext()) {
			System.out.println("Fatal Error! Wrong feature list!");
		}
	}

	public static void run() {

		String[] textPool = { "2012.q1.txt", "2012.q2.txt", "2012.q3.txt",
				"2012.q4.txt", };
		for (int i = 0; i < textPool.length; i++) {
			String dataFile = LinkerDriver.DATA_DIR + textPool[i];
			String resultFile = LinkerDriver.DATA_DIR + cleanDocName;
			DocReader dr = new DocReader(dataFile, resultFile);
		}
	}

	public static void searchPrev() {
		ArrayList<String> pool = new ArrayList<String>();
		String source = LinkerDriver.DATA_DIR + cleanDocName;
		MyFileReader mfr = new MyFileReader(source);
		String temp = null;
		StringBuilder sb = new StringBuilder();
		while ((temp = mfr.getNextLine()) != null) {
			if (temp.contains("<doc>")) {
				sb = new StringBuilder();
			} else if (temp.contains("</doc>")) {
				pool.add(sb.toString());
			} else {
				sb.append(temp + "\n");
			}
		}
		mfr.close();

		System.out.println("Read text pool done");

		ArrayList<String> target = new ArrayList<String>();
		String labeledText = LinkerDriver.DATA_DIR + "LabeledText.txt";
		mfr = new MyFileReader(labeledText);
		while ((temp = mfr.getNextLine()) != null) {
			target.add(MyStrUtil.getRidOfLabels(temp));
		}
		mfr.close();

		System.out.println("Read labeled text done");

		String outputFile = LinkerDriver.DATA_DIR + "completeText.txt";
		MyFileWriter mfw = new MyFileWriter(outputFile, true);
		String unrecognizedFile = LinkerDriver.DATA_DIR + "unrecognized.txt";
		MyFileWriter rest = new MyFileWriter(unrecognizedFile, true);

		Iterator<String> ltIter = target.iterator();
		int line = 0;

		while (ltIter.hasNext()) {
			line++;
			String text = ltIter.next();
			text = text.substring(0 > text.length() - 100 ? 0
					: text.length() - 100, text.length());
			Iterator<String> sIter = pool.iterator();
			int pos = 0;
			while (sIter.hasNext()) {
				String s = sIter.next();
				if (s.contains(text)) {
					s = MyStrUtil.getRidOf(s, "(微博)");
					s = MyStrUtil.getRidOf(s, "[微博]");
					s = MyStrUtil.getRidOf(s, "&nbsp;");
					mfw.write("<doc>\n" + s + "</doc>\n");
					break;
				}
				pos++;
			}
			if (pos < pool.size()) {
				pool.remove(pos);
			} else {
				rest.write(text + "\n");
			}
			if (line % 10 == 0) {
				System.out.println("line " + line + " is done");
			}
		}
		rest.close();
		mfw.close();
	}

	public static List<QueryContext> getQueryContext(AccountPool accountPool) {
		List<QueryContext> qcl = new ArrayList<QueryContext>();
		String source = LinkerDriver.DATA_DIR + "FormatInput.txt";
		MyFileReader mfr = new MyFileReader(source);
		String temp = null;
		List<String> list = new ArrayList<String>();
		while ((temp = mfr.getNextLine()) != null) {
			if (temp.contains("<QueryContext>")) {
				list.clear();
			} else if (temp.contains("</QueryContext>")) {
				if (list.size() > 0) {
					qcl.add(new QueryContext(list, accountPool));
				}
			} else {
				list.add(temp.trim());
			}
		}

		mfr.close();
		return qcl;
	}

	public static List<QueryContext> getQueryContextFromText(
			AccountPool accountPool) {
		List<QueryContext> qcl = new ArrayList<QueryContext>();
		String source = LinkerDriver.DATA_DIR + "LabeledText.txt";

		MyFileReader mfr = new MyFileReader(source);
		String temp = null;
		int userCount = 0, queryCount = 0;
		int linkedQuery = 0, unlinkedQuery = 0;
		int findCorrectQuery = 0;
		while ((temp = mfr.getNextLine()) != null) {
			QueryContext qc = new QueryContext(temp, accountPool);
			if (qc.queryList.size() == 0) {
				continue;
			}
			qcl.add(qc);
			for (int i = 0; i < qc.queryList.size(); i++) {
				userCount += qc.queryList.get(i).potential.size();
				if (qc.queryList.get(i).keyword.isLinked) {
					linkedQuery++;
				} else {
					unlinkedQuery++;
				}
				if (qc.queryList.get(i).findCorrect) {
					findCorrectQuery++;
				}
			}
			queryCount += qc.queryList.size();
		}
		System.out.println("There are " + linkedQuery + " linked queries, "
				+ unlinkedQuery + " unlinked queries");
		System.out.println("There are " + queryCount + " queries in total.");
		double avg = (double) userCount / (double) queryCount;
		System.out.println("There are " + avg + " candidates for each query");
		System.out.println(findCorrectQuery
				+ " queries find correct user in candidate list, with rate of "
				+ ((double) findCorrectQuery / queryCount));
		return qcl;
	}

	public static void saveQueryContextList(List<QueryContext> contextList) {
		Iterator<QueryContext> iter = contextList.iterator();
		String formatInput = LinkerDriver.DATA_DIR + "newFormatInput.txt";
		MyFileWriter mfw = new MyFileWriter(formatInput, true);
		while (iter.hasNext()) {
			iter.next().recordInfo(mfw);
		}
		mfw.close();
	}

}
